__author__ = 'Knight'

import time
import urllib.request
import urllib.parse
import urllib.error
import log
from html.parser import HTMLParser
import gzip
import traceback

class Parse(HTMLParser):

    def __init__(self):
        HTMLParser.__init__(self)

    @staticmethod
    def get_dict_attrs(attrs):
        dict_attrs = {}
        for (k, v) in attrs:
            dict_attrs[k] = v
        return dict_attrs

    def get_v_by_k(self, attrs, k):
        dict_attrs = self.get_dict_attrs(attrs)
        return dict_attrs.get(k)

    def is_kv_in_attrs(self, attrs, k, v):
        dict_attrs = self.get_dict_attrs(attrs)
        if v == dict_attrs.get(k):
            return True
        else:
            return False

    def is_k_in_attrs(self, attrs, k):
        dict_attrs = self.get_dict_attrs(attrs)
        if k in dict_attrs.keys():
            return True
        return False

    def is_class_v_in_attrs(self, attrs, v):
        return self.is_kv_in_attrs(attrs, 'class', v)

    def set_start(self, tag, attrs, s_tag=None, s_k=None, s_v=None):
        tf_tag, tf_kv = True, True
        if s_tag is not None:
            tf_tag = True if tag == s_tag else False
        if s_k is not None and s_v is not None:
            tf_kv = self.is_kv_in_attrs(attrs, s_k, s_v)
        elif s_k is not None and s_v is None:
            tf_kv = self.is_k_in_attrs(attrs, s_k)
        elif s_k is None and s_v is not None:
            tf_kv = self.is_class_v_in_attrs(attrs, s_v)
        if tf_tag and tf_kv:
            return True
        return False

class RedirectError(Exception):

    def __init__(self, url, re_url):
        Exception.__init__(self)
        self.url = url
        self.re_url = re_url

    def __str__(self):
        err = 'url redirect'.center(40, '-')
        err += '\nfrom : [{}]\n to  : [{}]'.format(self.url, self.re_url)
        return err

class Proxy:

    @staticmethod
    def get_data(data):
        """
        把输入的data进行格式转化 data类型必须为 dict str
        :param data: 传入的post请求data
        :return: 格式化好的post请求data
        """
        assert isinstance(data, (str, dict))
        if isinstance(data, str):
            data = data.encode(encoding='utf-8')
        elif isinstance(data, dict):
            data = urllib.parse.urlencode(data).encode(encoding='utf-8')
        return data

    @staticmethod
    def get_request(username, password):
        # 代理服务器信息
        proxy_host, proxy_port = "proxy.abuyun.com", "9010"
        # 代理隧道验证信息
        proxy_user = "H49SP8W65524RY2P" if username is None else username
        proxy_pass = "45C1DEC7B501890A" if password is None else password
        # proxy_user, proxy_pass = "H49SP8W65524RY2P", "45C1DEC7B501890A"

        proxy_meta = 'http://{}:{}@{}:{}'.format(proxy_user, proxy_pass, proxy_host, proxy_port)
        proxy_handler = urllib.request.ProxyHandler({
            "http": proxy_meta,
            "https": proxy_meta,
        })
        opener = urllib.request.build_opener(proxy_handler)
        urllib.request.install_opener(opener)

    def __init__(self, url, is_proxy=True, headers=None, fmt=None, data=None, timeout=None,
                 username=None, password=None, logger=None):
        self.logger = logger if logger else log.get_stream_log()
        self.url = url
        self.fmt = 'utf-8' if fmt is None else fmt
        self.headers = {} if headers is None else headers
        self.headers['Proxy-Switch-Ip'] = 'yes'
        self.data_in = data
        self.data = self.get_data(data) if data else data
        self.timeout = timeout if timeout else 10
        self.fmt = fmt if fmt else 'utf8'
        self.is_proxy = is_proxy
        self.info = None
        self.geturl = None
        self.code = None
        if self.is_proxy:
            self.get_request(username, password)

    def is_redirect(self):
        return self.url != self.geturl

    def get_geturl(self):
        return self.geturl

    def get_info(self):
        return self.info

    def get_code(self):
        return self.code

    def get_fmt(self):
        try:
            ct = self.info['Content-Type']
            idx = str(ct).index('charset=')
            if idx > 0:
                fmt = ct[idx + 8:]
                '自动截获解码格式 {}'.format(fmt)
                if fmt == 'gb2312':
                    return 'gbk'
                return fmt
        except Exception as e:
            raise Exception()

    def get_page_proxy_inner(self):
        for i in range(3):
            try:
                req = urllib.request.Request(self.url, headers=self.headers, data=self.data)
                response = urllib.request.urlopen(req, data=self.data, timeout=self.timeout)
                self.info = response.info()
                self.geturl = response.geturl()
                self.code = response.getcode()
                if self.fmt == 'auto':
                    self.fmt = self.get_fmt()
                res = response.read()
                if self.headers.get('Accept-Encoding') is not None:
                    try:
                        res = gzip.decompress(res)
                    except OSError as e:
                        pass
                res = res.decode(encoding=self.fmt, errors='ignore')
                # 代理proxy检测
                if str(res).lower().find('too many requests') >= 0:
                    self.logger.warning('too many requests - {}'.format(i + 1))
                    time.sleep(1)
                    continue
                return res
            except Exception:
                raise
        raise Exception('too many requests')

    def get_page_local_inner(self):

        try:
            request = urllib.request.Request(self.url, headers=self.headers, data=self.data)
            response = urllib.request.urlopen(request, data=self.data, timeout=self.timeout)
            self.info = response.info()
            self.geturl = response.geturl()
            if self.fmt == 'auto':
                self.fmt = self.get_fmt()
            res = response.read()
            if self.headers.get('Accept-Encoding') is not None:
                try:
                    res = gzip.decompress(res)
                except OSError as e:
                    pass
            res = res.decode(encoding=self.fmt, errors='ignore')
            return res
        except Exception:
            raise

    def get_page(self, repeat_idx=4, is_raise=False):
        tmp_exception = []
        for i in range(repeat_idx):
            # 重试休眠
            if i > 0:
                self.logger.warning('retry: {} time.sleep {}'.format(i, i * 10))
                time.sleep(i * 10)

            try:
                if self.is_proxy:
                    res = self.get_page_proxy_inner()
                else:
                    res = self.get_page_local_inner()
                return res

            except Exception as e:
                if is_raise and i == repeat_idx - 1:
                    raise
                self.logger.info(e.__class__.__name__)
                self.logger.info(traceback.format_exc())
                tmp_exception.append(e.__class__.__name__)
                err = 'error url: {}'.format(self.url)
                if self.data_in is not None:
                    err += '\n data_in: {}'.format(self.data_in)
                self.logger.info(err)
            except UnicodeDecodeError:
                try:
                    fmt_suggest = self.get_fmt()
                    self.logger.info(
                        'decode error with: "{}" suggest "{}" by response.info'.format(self.fmt, fmt_suggest)
                    )
                    tmp_exception.append('UnicodeDecodeError with {} suggest {}'.format(self.fmt, fmt_suggest))
                except Exception as e:
                    self.logger.info(
                        'decode error with: "{}" suggest by response.info get failed'
                        '{}'.format(self.fmt, e.args)
                    )
                    tmp_exception.append('UnicodeDecodeError with {} suggest failed'.format(self.fmt))

        err = 'err type'.center(40, '-') + '\n'
        err += ''.join(map(lambda x: x + '\n', tmp_exception))
        err += 'err info'.center(40, '-') + '\n'
        err += 'url: {}'.format(self.url) + '\n'
        if self.data_in is not None:
            err += 'data_in: {}'.format(self.data_in)
        raise Exception(err)
